# ---------------------------------------------------------------------------
# Author: Min Feng
# Date: 01/04/08
# saves the list of datasets in repository folder.
#----------------------------------------------------------------------------

import configure
import ftplib

import os
import re

def download(f, t, l):
    _m = re.match('/dmf/pds/\w+/\w+\d/\d+/\d+/\d+/(\w{3}\d+\.tar)', l)
    if _m:
        _l = _m.group(1)
        _file = open(os.path.join(t, _l), 'wb')
        try:
            print '\t', _l,
            f.retrbinary('RETR %s' % l, _file.write)
            _file.close()

            print 'ok'
            return True
        except Exception, err:
            print 'failed', err
            _file.close()
            os.remove(os.path.join(t, _l))
    return False

def buildDataList():
    _list = []
    for _root, _dirs, _files in os.walk(_conf.getPathRepository()):
        for _f in _files:
            _m = re.match('\w{3}(\d+)\.zip', _f)
            _list.append(_m.group(1))

    return _list

def listDir(f, l):
    try:
        print l
        return f.nlst(l)
    except Exception, err:
        return []

def listFiles(f, l):
    try:
        return f.nlst(l)
    except Exception, err:
        return []

def filterList(l):
    _dataList = buildDataList()
    _list2 = {}
    for _l in l:
        _m = re.match('/dmf/pds/\w+/\w+\d/\d+/\d+/\d+/(\w{3})(\d+)\.tar', _l)
        if _m:
            if _m.group(1) == 'MBZ' or _m.group(1) == 'NZT':
                _k = _m.group(2)
                if (_k not in _list2) and (_k not in _dataList):
                    _list2[_k] = _l
                else:
                    print 'exist', _l

    print 'After filtering', len(_list2.keys())
    return _list2.values()
    
def downloadList(f, p, l):
    print 'Start to download...'
    for _l in l:
        print 'download', _l,
        if download(f, p, _l) == False:
            _failure.append(_l + '\n')
            print 'failed'
        else:
            _success.append(_l + '\n')

_conf = configure.configure()
_f = ftplib.FTP(_conf.getString('ftp.server'))
_f.login(_conf.getString('ftp.username'), _conf.getString('ftp.password'))
_f.set_pasv(False)

_success = []
_failure = []

_types = ['/dmf/pds/mtbs', '/dmf/pds/mrlc92', '/dmf/pds/mrlc2k']
_satellites = ['landsat7', 'landsat5', 'landsat4']

print 'login success.'
try:
    _list = []
    for _l in open(_conf.getString('ftp.file.path.row'), 'r'):
        _l = _l.strip()
        _path, _row = _l.split(',')

        for _t in _types:
            for _s in _satellites:
                for _d in listDir(_f, _t + '/' + _s + '/' + str(_path) + '/' + str(_row)):
                    _list.extend(listFiles(_f, _d))

    print 'Found', len(_list), 'files'
    downloadList(_f, _conf.getString('ftp.path.download'), filterList(_list))

finally:
    _f.close()

open(_conf.getString('ftp.log.success'), 'w').writelines(_success)
open(_conf.getString('ftp.log.failure'), 'w').writelines(_failure)
